#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Combot数据抓取脚本
用于批量获取所有语言的Telegram群组数据
"""

data = [
    {'language':'all','count':42647,'name':'Global'},
    {'language':'en','count':12525,'name':'English'},
    {'language':'ru','count':8151,'name':'Русский'},
    {'language':'tr','count':2922,'name':'Türkçe'},
    {'language':'zh','count':2554,'name':'中文'},
    {'language':'id','count':1686,'name':'Indonesia'},
    {'language':'vi','count':1420,'name':'Tiếng Việt'},
    {'language':'es','count':1153,'name':'Español'},
    {'language':'uz','count':1152,'name':'Oʻzbek'},
    {'language':'hi','count':946,'name':'हिंदी'},
    {'language':'pt','count':707,'name':'Português'},
    {'language':'ko','count':616,'name':'한국어'},
    {'language':'ar','count':607,'name':'العربية'},
    {'language':'az','count':600,'name':'Azərbaycan'},
    {'language':'fa','count':544,'name':'فارسی'},
    {'language':'it','count':498,'name':'Italiano'},
    {'language':'uk','count':455,'name':'Українська'},
    {'language':'ms','count':317,'name':'Malay'},
    {'language':'bn','count':264,'name':'বাংলা'},
    {'language':'ta','count':257,'name':'தமிழ்'},
    {'language':'fr','count':241,'name':'Français'},
    {'language':'de','count':232,'name':'Deutsch'},
    {'language':'ja','count':194,'name':'日本語'},
    {'language':'si','count':134,'name':'සිංහල'},
    {'language':'am','count':112,'name':'አማርኛ'},
    {'language':'my','count':107,'name':'မြန်မာစာ'},
    {'language':'ig','count':106,'name':'Asụsụ Igbo'},
    {'language':'pl','count':103,'name':'Polski'},
    {'language':'fil','count':96,'name':'Wikang Filipino'},
    {'language':'ml','count':92,'name':'മലയാളം'},
    {'language':'tg','count':86,'name':'Тоҷикӣ'},
    {'language':'so','count':85,'name':'Af-Soomaali'},
    {'language':'he','count':79,'name':'עברית'},
    {'language':'jv','count':67,'name':'ꦧꦱꦗꦮ'},
    {'language':'nl','count':66,'name':'Nederlands'},
    {'language':'ky','count':64,'name':'Кыргызча'},
    {'language':'ro','count':61,'name':'Română'},
    {'language':'ur','count':51,'name':'اُردُو'},
    {'language':'th','count':50,'name':'ภาษาไทย'},
    {'language':'kk','count':47,'name':'Казақша'},
    {'language':'sw','count':46,'name':'Kiswahili'},
    {'language':'mr','count':44,'name':'मराठी'},
    {'language':'te','count':37,'name':'తెలుగు'},
    {'language':'el','count':25,'name':'Νέα Ελληνικά'},
    {'language':'sv','count':23,'name':'Svenska'},
    {'language':'hu','count':18,'name':'Magyar nyelv'},
    {'language':'km','count':16,'name':'ខេមរភាសា'},
    {'language':'ne','count':15,'name':'नेपाली भाषा'},
    {'language':'ha','count':15,'name':'هَرْشٜن هَوْس'},
    {'language':'ca','count':13,'name':'Català'},
    {'language':'kn','count':11,'name':'ಕನ್ನಡ'},
    {'language':'lt','count':10,'name':'lt'},
]

import requests
import json
import time
import pandas as pd
import base64
import os
from datetime import datetime

base_url = "https://combot.org"

def get_image_type(image_data):
    """
    判断图像数据类型
    """
    if not image_data:
        return '无头像'
    elif image_data.startswith('data:image/'):
        return '完整data URL'
    elif image_data.startswith('/9j/') or image_data.startswith('iVBOR'):
        return 'Base64图片'
    else:
        return '相对路径'

def get_last_page(json_folder, language):
    """
    获取某种语言已抓取的最后一页
    """
    existing_files = [f for f in os.listdir(json_folder) if f.startswith(f"{language}_") and f.endswith('.json')]
    if not existing_files:
        return 0
    
    # 提取页数并找到最大值
    page_numbers = []
    for file in existing_files:
        try:
            page_num = int(file.split('_')[1].split('.')[0])
            page_numbers.append(page_num)
        except:
            continue
    
    return max(page_numbers) if page_numbers else 0

def scrape_all():
    # 创建数据文件夹
    data_folder = "combot_data"
    json_folder = os.path.join(data_folder, "json")
    if not os.path.exists(data_folder):
        os.makedirs(data_folder)
    if not os.path.exists(json_folder):
        os.makedirs(json_folder)
    
    all_processed_data = []  # 存储所有语言的数据
    
    for item in data:
        language = item['language']
        total_count = item['count']
        name = item['name']
        
        # 检查是否已经抓取过，从断点继续
        last_page = get_last_page(json_folder, language)
        
        # 估算总页数
        estimated_total_pages = (total_count + 49) // 50  # 向上取整
        
        if last_page >= estimated_total_pages:
            print(f"跳过 {name} ({language}) - 已完成抓取 (共 {last_page} 页)")
            continue
        elif last_page > 0:
            print(f"检测到 {name} ({language}) 已抓取到第 {last_page} 页，从第 {last_page + 1} 页继续 (预计总共 {estimated_total_pages} 页)")
            start_page = last_page + 1
            start_offset = last_page * 50
        else:
            print(f"开始抓取 {name} ({language}) - 共 {total_count} 个群组 (预计 {estimated_total_pages} 页)")
            start_page = 1
            start_offset = 0
        
        all_data = []
        limit = 50  # 每次请求50条数据
        offset = start_offset
        page_num = start_page
        
        # 多页请求逻辑
        while offset < total_count:
            print(f"  获取 {language} 数据: 第{page_num}页 offset={offset}, limit={limit}")
            
            url = f"{base_url}/api/chart/{language}"
            params = {
                'limit': limit,
                'offset': offset
            }
            
            try:
                response = requests.get(url, params=params, timeout=10)
                response.raise_for_status()
                
                if response.status_code == 204:
                    break  # 没有更多数据
                
                batch_data = response.json()
                if not batch_data:
                    break
                
                # 保存每次请求的JSON数据
                json_filename = os.path.join(json_folder, f"{language}_{page_num}.json")
                try:
                    with open(json_filename, 'w', encoding='utf-8') as f:
                        json.dump(batch_data, f, ensure_ascii=False, indent=2)
                    print(f"    已保存: {json_filename} ({len(batch_data)} 条数据)")
                except Exception as e:
                    print(f"    保存JSON失败: {e}")
                
                all_data.extend(batch_data)
                offset += limit  # 固定步长递增：0, 50, 100, 150...
                page_num += 1  # 页数递增
                
                # 添加延迟避免请求过快
                time.sleep(0.3)
                
                # 如果返回的数据少于limit，说明已经到最后一页
                if len(batch_data) < limit:
                    break
                    
            except Exception as e:
                print(f"  请求失败: {e}")
                break
        
        print(f"  {language} 完成，实际获取 {len(all_data)} 个群组")
        
        if all_data:
            # 处理数据
            processed_data_for_lang = []
            for group in all_data:
                processed_group = {
                    '语言': name,
                    '语言代码': language,
                    '群组标题': group.get('t', ''),
                    '用户名': group.get('u', ''),
                    '成员数': group.get('s', 0),
                    '排名': group.get('p', 0),
                    '位置变化': group.get('pc', ''),
                    '语言标识': group.get('l', ''),
                    '年龄': group.get('a', ''),
                    '头像数据': '有头像' if group.get('i') else '无头像',
                    '头像类型': get_image_type(group.get('i', '')),
                    '抓取时间': datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                }
                all_processed_data.append(processed_group)
                processed_data_for_lang.append(processed_group)
            
            # 保存单个语言的JSON备份到文件夹
            json_filename = os.path.join(data_folder, f"combot_data_{language}.json")
            try:
                with open(json_filename, 'w', encoding='utf-8') as f:
                    json.dump(all_data, f, ensure_ascii=False, indent=2)
                print(f"  JSON备份已保存到 {json_filename}")
            except Exception as e:
                print(f"  保存JSON失败: {e}")
            
            # 实时保存单个语言的Excel文件
            excel_filename = os.path.join(data_folder, f"combot_data_{language}.xlsx")
            try:
                df_lang = pd.DataFrame(processed_data_for_lang)
                df_lang.to_excel(excel_filename, index=False, engine='openpyxl')
                print(f"  Excel已保存到 {excel_filename}")
            except Exception as e:
                print(f"  保存Excel失败: {e}")
            
            # 实时更新总Excel文件（防止程序中断丢失数据）
            if all_processed_data:
                try:
                    df_all = pd.DataFrame(all_processed_data)
                    df_all = df_all.sort_values('成员数', ascending=False)
                    temp_excel = os.path.join(data_folder, "combot_all_data_temp.xlsx")
                    df_all.to_excel(temp_excel, index=False, engine='openpyxl')
                    print(f"  临时总文件已更新: {temp_excel}")
                except Exception as e:
                    print(f"  更新临时总文件失败: {e}")
        
        print()
    
    # 将所有数据保存到一个Excel文件
    if all_processed_data:
        print("正在保存所有数据到Excel文件...")
        df = pd.DataFrame(all_processed_data)
        
        # 按成员数排序
        df = df.sort_values('成员数', ascending=False)
        
        # 保存到数据文件夹
        excel_filename = os.path.join(data_folder, f"combot_all_data_{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx")
        
        try:
            df.to_excel(excel_filename, index=False, engine='openpyxl')
            print(f"所有数据已保存到: {excel_filename}")
            print(f"总计 {len(df)} 个群组")
            
            # 统计各语言数量
            lang_stats = df['语言'].value_counts()
            print("\n各语言群组数量统计:")
            for lang, count in lang_stats.items():
                print(f"  {lang}: {count} 个群组")
                
        except Exception as e:
            print(f"保存Excel失败: {e}")
    else:
        print("没有获取到任何数据")
        
        


def main():
    """
    主函数
    """
    print("=== Combot数据抓取开始 ===")
    
    try:
        scrape_all()
        print("=== 抓取完成 ===")
        
    except KeyboardInterrupt:
        print("\n程序被用户中断")
    except Exception as e:
        print(f"程序执行出错: {e}")

if __name__ == "__main__":
    main()
